# -*- coding:utf-8 -*-
# --author： jingfeng 
# time: 2019/3/9


from requests import Request,Session
from db import RedisQueue
import requests
from pyquery import PyQuery as pq
from urllib.parse import urlencode
from save_mongdb import MyMongodb


class WeiXinRequests(Request):

    def __init__(self, url, callback, method='GET', headers=None, need_proxy=False, fail_time=0, timeout=TIMEOUT):
        Request.__init__(self, method, url, headers)
        self.callback = callback
        self.need_proxy = need_proxy
        self.fail_time = 20
        self.timeout = 20



class Spider():
    base_url = 'https://weixin.sogou.com/weixin'
    keyword = '养生堂'
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cache-Control': 'max-age=0',
        'Connection': 'keep-alive',
        'Cookie': 'IPLOC=CN6101; SUID=27D82075541C940A000000005C83646E; SUV=1552114800443087; ABTEST=7|1552114803|v1; SNUID=3DC2396C1A1C99B9F99C83691AE4CB25; weixinIndexVisited=1; sct=1; JSESSIONID=aaamUELIlbwI6boyuxZKw; ppinf=5|1552115152|1553324752|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZToxODolRTUlQkYlOTclRTklQjklOEZ8Y3J0OjEwOjE1NTIxMTUxNTJ8cmVmbmljazoxODolRTUlQkYlOTclRTklQjklOEZ8dXNlcmlkOjQ0Om85dDJsdU5RSXplN3g3QVQ2Y3U5cXJzcUlSV1FAd2VpeGluLnNvaHUuY29tfA; pprdig=bGv_TFYF46bfWSXidz1hbfMyyZuAvoE_WUltHY8mv8gCWQ_eAZFi_-r3zE4lC09fQQd4c6uKL2pjSM7qFNkdBDz-JNH_ABHGWGAVx2KA2I4F7RtdQANPud2tKPwQJb-cAWDWjxHZW7LlIL3vu0qSNHSRLPFTc38i_z2wy15hOWI; sgid=13-34355091-AVyDZdDb0eNuJwSotsWup5c; ppmdig=155211515300000054bc12ea64a7e1aaeff4e289820c4238',
        'Host': 'weixin.sogou.com',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'}
    session = Session()
    queue = RedisQueue()
    my_mongodb = MyMongodb()

    def get_proxy(self):
        """
        得到ip
        """
        try:
            response = requests.get('http://127.0.0.1:5555/random')
            if response.status_code == 200:
                print(response.text)
                return response.text
            return None
        except:
            return None

    def start(self):
        """
        初始化工作
        """
        # 全局更新Headers
        self.session.headers.update(self.headers)
        start_url = self.base_url + '?' + urlencode(
                {'query': self.keyword, 'type': 2})
        weixin_request = WeiXinRequests(url=start_url,
                                        callback=self.parse_index,
                                        need_proxy=True)
        # 调度第一个请求
        self.queue.add(weixin_request)

    def parse_index(self, response):
        """
        解析索引页
        :param response: 响应
        :return: 新的响应
        """
        doc = pq(response.text)
        items = doc('.news-box .news-list li .txt-box h3 a').items()
        for item in items:
            url = item.attr('href')
            weixin_request = WeiXinRequests(url=url, callback=self.parse_detail)
            yield weixin_request
        next = doc('#sogou_next').attr('href')
        if next:
            url = self.base_url + str(next)
            weixin_request = WeiXinRequests(url=url, callback=self.parse_index,
                                           need_proxy=True)
            yield weixin_request

    def parse_detail(self, response):
        """
        解析详情页
        :param response: 响应
        :return: 微信公众号文章
        """
        doc = pq(response.text)
        data = {
            'title': doc('.rich_media_title').text(),
            'content': doc('.rich_media_content').text(),
            'date': doc('#post-date').text(),
            'nickname': doc('#js_profile_qrcode > div > strong').text(),
            'wechat': doc(
                '#js_profile_qrcode > div > p:nth-child(3) > span').text()
        }
        yield data

    def request(self, weixin_request):
        """
        执行请求
        :param weixin_request: 请求
        :return: 响应
        """
        try:
            if weixin_request.need_proxy:
                proxy = self.get_proxy()
                if proxy:
                    proxies = {
                        'http': 'http://' + proxy,
                        'https': 'https://' + proxy
                    }
                    return self.session.send(weixin_request.prepare(),
                                             timeout=weixin_request.timeout,
                                             allow_redirects=False,
                                             proxies=proxies)
            return self.session.send(weixin_request.prepare(),
                                     timeout=weixin_request.timeout,
                                     allow_redirects=False)
        except:
            print('错误')
            return False

    def error(self, weixin_request):
        """
        错误处理
        :param weixin_request: 请求
        :return:
        """
        weixin_request.fail_time = weixin_request.fail_time + 1
        print('Request Failed', weixin_request.fail_time, 'Times',
              weixin_request.url)
        if weixin_request.fail_time < 20:
            self.queue.add(weixin_request)

    def schedule(self):
        """
        调度请求
        :return:
        """
        while not self.queue.empty():
            weixin_request = self.queue.pop()
            callback = weixin_request.callback
            print('Schedule', weixin_request.url)
            response = self.request(weixin_request)
            if response and response.status_code in [200]:
                results = list(callback(response))
                if results:
                    for result in results:
                        print('New Result', type(result))
                        if isinstance(result, WeiXinRequests):
                            self.queue.add(result)
                        if isinstance(result, dict):
                            self.my_mongodb.insert(dict)

                else:
                    self.error(weixin_request)
            else:
                self.error(weixin_request)

    def run(self):
        """
        入口
        :return:
        """
        self.start()
        self.schedule()
